# RUN THIS CELL TO IMPORT YOUR DATA. 

# Load dataset into a dataframe
df0 = pd.read_csv("HR_capstone_dataset_Salifort.csv")

df = df0.copy()

# Display first few rows of the dataframe
df.head(10)

# Gather basic information about the data
df.shape

df.info()

# Gather descriptive statistics about the data
df.describe()

# Display all column names
df.columns

# Check for missing values
df.isna().sum()

# Check for duplicates
df.duplicated().sum()

# Inspect some rows containing duplicates as needed
df[df.duplicated()]

# 1. T-test (satisfaction_level ~ left) + manual Cohen's d
group_left = df[df['left'] == 1]['satisfaction_level']
group_stayed = df[df['left'] == 0]['satisfaction_level']

ttest, p_value = stats.ttest_ind(group_left, group_stayed, equal_var=False)
print(f"T-test = {ttest:.4f}, p-value = {p_value:.6f}\n")

df_ttest = df.copy()
t_test_metric = df_ttest.drop(columns=['left', 'department', 'salary'])

metrics = {}
for metric in t_test_metric.columns:
        group_left = df_ttest[df_ttest['left'] == 1][metric]
        group_stayed = df_ttest[df_ttest['left'] == 0][metric]
           # Convert to numeric in case of hidden strings
        group_left = pd.to_numeric(group_left, errors='coerce')
        group_stayed = pd.to_numeric(group_stayed, errors='coerce')
        ttest, p_value = stats.ttest_ind(group_left, group_stayed, equal_var=False)
        metrics[metric] = {'t_test': ttest, 'p_value': p_value}

# Convert dictionary to DataFrame
metrics_df = pd.DataFrame.from_dict(metrics, orient='index')

# Reset index to turn metric names into a column
metrics_df = metrics_df.reset_index().rename(columns={'index': 'metric'})

# Sort by p-value
metrics_df = metrics_df.sort_values(by='p_value')

# Optional: round values for readability
metrics_df = metrics_df.round({'t_test': 3})

metrics_df

# Create a contingency table
dept_contingency = pd.crosstab(df['department'], df['left'])

# Run chi-squared test
chi2_dept, p_dept, _, _ = chi2_contingency(dept_contingency)
print(f"Chi-squared test for department vs left: p-value = {p_dept}")

salary_contingency = pd.crosstab(df['salary'], df['left'])

chi2_salary, p_salary, _, _ = chi2_contingency(salary_contingency)
print(f"Chi-squared test for salary vs left: p-value = {p_salary}")

metrics_df = pd.concat([
    metrics_df,
    pd.DataFrame([
        {'metric': 'department (chi2)', 't_test': None, 'p_value': p_dept},
        {'metric': 'salary (chi2)', 't_test': None, 'p_value': p_salary}
    ])
])

# Sort again by p-value
metrics_df = metrics_df.sort_values(by='p_value').reset_index(drop=True)

print(metrics_df)

metrics_df['log_p_value'] = -np.log10(metrics_df['p_value'])

# Plot
sns.barplot(data=metrics_df, x='metric', y='log_p_value')
plt.xticks(rotation=45, ha='right')
plt.ylabel('log10(p-value)')
plt.title('Log-scaled P-values for Metrics')
plt.tight_layout()
plt.show()

# Resignation rate by salary
fig, ax = plt.subplots(1, 2, figsize=(15, 6))
sns.barplot(x='salary', y='left', data=df, ax=ax[0])
ax[0].set_title("Turnover Rate by Salary")

# Resignation rate by department
sns.barplot(x='department', y='left', data=df, ax=ax[1])
ax[1].tick_params(axis='x', rotation=45)
ax[1].set_title("Turnover Rate by Department")

plt.tight_layout()
plt.show()

fig, ax = plt.subplots(1, 2, figsize = (22,8))

# Create boxplot showing `average_monthly_hours` distributions for `number_project`, comparing employees who stayed versus those who left
sns.boxplot(data=df, x='average_monthly_hours', y='number_project', hue='left', orient="h", ax=ax[0])
ax[0].invert_yaxis()
ax[0].set_title('Monthly hours by number of projects', fontsize='14')

# Create histogram showing distribution of `number_project`, comparing employees who stayed versus those who left
sns.histplot(data=df, x='number_project', hue='left', multiple='dodge', shrink=2, ax=ax[1])
ax[1].set_title('Number of projects histogram', fontsize='14')

# Display the plots
plt.show()

# Set figure and axes
fig, ax = plt.subplots(1, 2, figsize = (22,8))

# Define short-tenured employees
tenure_short = df[df['tenure'] < 7]

# Define long-tenured employees
tenure_long = df[df['tenure'] > 6]

# Plot short-tenured histogram
sns.histplot(data=tenure_short, x='tenure', hue='salary', discrete=1, 
             hue_order=['low', 'medium', 'high'], multiple='dodge', shrink=.5, ax=ax[0])
ax[0].set_title('Salary histogram by tenure: short-tenured people', fontsize='14')

# Plot long-tenured histogram
sns.histplot(data=tenure_long, x='tenure', hue='salary', discrete=1, 
             hue_order=['low', 'medium', 'high'], multiple='dodge', shrink=.4, ax=ax[1])
ax[1].set_title('Salary histogram by tenure: long-tenured people', fontsize='14')

# Take only numerical metrics
metrics = df.select_dtypes(include='number').columns

# Take off binary metrics from numerical metrics
metrics_non_binary = [col for col in metrics if df[col].nunique() > 2]

for metric in metrics_non_binary:
# Determine the number of rows containing outliers
    perc75 = df[metric].quantile(0.75)
    perc25 = df[metric].quantile(0.25)
    iqr = perc75 - perc25
    upper_limit = perc75 + 1.5 * iqr
    lower_limit = perc25 - 1.5 * iqr
    outliers =  df[(df[metric] > upper_limit) |  (df[metric] < lower_limit)]
    print(f"\nMetric: {metric}")
    print("Lower limit:", lower_limit)
    print("Upper limit:", upper_limit)
    print(f"Number of outliers in {metric}: {len(outliers)}")

# Get numbers of people who left vs. stayed
status_mapping = {
    0:'Stayed',
    1:'Left'
}

print("\nNumber of People Who Left vs. Stayed:")
print(df['left'].map(status_mapping).value_counts().astype(str))


# Get percentages of people who left vs. stayed
print("\nPercentage of People Who Left vs. Stayed:")
print(df['left'].map(status_mapping).value_counts(normalize=True).round(2).astype(str)+"%")

# Create a plot as needed
sns.boxplot(x=df['satisfaction_level'])
plt.title('Satisfaction Level Distribution')
plt.xlabel('Satisfaction Level')

# Create a plot as needed
sns.boxplot(x=df['average_monthly_hours'])
plt.title('Average Monthly Hours Boxplot')
plt.xlabel('Average Monthly Hours')

# Create a plot as needed
sns.boxplot(x=df['tenure'])
plt.title('Tenure Boxplot')
plt.xlabel('Tenure (years)')

# Create a plot as needed
counts = df['left'].value_counts()
labels = ['Stayed', 'Left']

plt.pie(counts, labels=labels, autopct='%1.1f%%', startangle=90)
plt.title('Distribution of Left Employees')
plt.show()

fig, ax = plt.subplots(1, 2, figsize=(12,6))
sns.countplot(data=df, x='promotion_last_5years', hue='left', ax=ax[0])
ax[0].set_title('Left vs Promotion Last 5 Years')
ax[0].legend(title='Left', labels=['Stayed', 'Left'])



sns.boxplot(data=df, x='tenure', y='number_project', ax=ax[1])
ax[1].set_title('Tenure vs Number of Projects')
ax[1].set_xlabel('Tenure (years)')
ax[1].set_ylabel('Number of Projects')
plt.xticks(rotation=45)
plt.tight_layout()

# Get numbers of people who left vs. stayed
status_mapping = {
    0:'Not promoted',
    1:'Promoted'
}

print("\nNumber of People Promoted vs Not Promoted:")
print(df['promotion_last_5years'].map(status_mapping).value_counts().to_string())


# Get percentages of people who left vs. stayed
print("\nPercentage of People Promoted vs Not Promoted:")
print(df['promotion_last_5years'].map(status_mapping).value_counts(normalize=True).round(2).astype(str)+"%")

# Create a plot as needed
plt.figure(figsize=(20,12))
sns.histplot(data = df, x = 'department', hue = 'left', multiple = 'dodge', shrink = 0.8)
plt.xlabel('Department', fontsize=14)
plt.xticks(fontsize=16)
plt.yticks(fontsize=16)
plt.legend(title='Left', labels=['Stayed', 'Left'])
plt.ylabel('Number of Employees', fontsize=14)
plt.xticks(rotation=45)
plt.title('Distribution of Employees by Department', fontsize=20)

# Create a plot as needed
df['department'].value_counts()

plt.figure(figsize=(20,12))
plt.xticks(fontsize=18)
plt.yticks(fontsize=16)
plt.xlabel('Satisfaction level', fontsize=16)
sns.histplot(x ='satisfaction_level', hue = 'salary', multiple='stack', data=df)
plt.title('Satisfaction Level by Salary', fontsize=20)
legend = plt.gca().get_legend()
legend.get_title().set_fontsize(18)
for text in legend.get_texts():
    text.set_fontsize(16)

df['salary'].value_counts()

df_sat_cat = df.copy()
df_sat_cat['satisfaction_cat'] = pd.cut(
    df['satisfaction_level'],
    bins=[0, 0.4, 0.7, 1.0],
    labels=['Low', 'Medium', 'High']
)
# - pd.cut() → This function bins the values into predefined ranges
# - bins=[0, 0.4, 0.7, 1.0] → Defines the intervals
# - 0 to 0.4 → Assigned to 'Low'.
# - 0.4 to 0.7 → Assigned to 'Medium'.
# - 0.7 to 1.0 → Assigned to 'High'.

satisfaction_counts = df_sat_cat['satisfaction_cat'].value_counts()

plt.figure(figsize=(8, 8))
plt.pie(satisfaction_counts, labels=satisfaction_counts.index, autopct='%1.1f%%', startangle=140)
plt.title('Satisfaction Level Categories')
plt.show()

fig, ax  = plt.subplots(1, 2, figsize=(20, 8))
sns.countplot(data=df, x='salary', hue='left', ax=ax[0])
ax[0].set_title('Leaving Employees by Salary Level')
ax[0].legend(title='Left', labels=['Stayed', 'Left'])
ax[0].set_xlabel('Salary Level')
ax[0].set_ylabel('Number of Employees')

sns.histplot(data=df, x='satisfaction_level', hue='left', multiple='stack', ax=ax[1])
ax[1].set_title('Satisfaction Level vs Leaving Employees')
ax[1].legend(title='Left', labels=['Stayed', 'Left'])
ax[1].set_xlabel('Satisfaction Level')
ax[1].set_ylabel('Number of Employees')

sns.scatterplot(data=df, x='average_monthly_hours', y='satisfaction_level', hue='left', alpha=0.4)
plt.axvline(x=166.67, color='orange', label='166.67 hrs./mo.', ls='--')
plt.title('Average Monthly Hours vs Satisfaction Level')

plt.figure(figsize=(16, 9))
sns.scatterplot(data=df, x='average_monthly_hours', y='last_evaluation', hue='left', alpha=0.4)
plt.axvline(x=166.67, color='orange', label='166.67 hrs./mo.', ls='--')
plt.legend(labels=['166.67 hrs./mo.', 'left', 'stayed'])
plt.title('Monthly hours by last evaluation score', fontsize='16')

# Create a new dataframe with only numeric columns
df_corr = df.select_dtypes(include='number') 

corr = df_corr.corr()
corr_target_sorted = corr.sort_values(by='left', ascending=False)

# Reorder rows and columns
ordered_corr = corr.loc[corr_target_sorted.index, corr_target_sorted.index]

# Plot with mask
mask = np.triu(np.ones_like(ordered_corr, dtype=bool))
plt.figure(figsize=(12, 10))
sns.heatmap(ordered_corr, mask=mask, annot=True, cmap='coolwarm', fmt='.2f')
plt.title('Sorted Correlation Matrix')
plt.show()

# Copy the dataframe
df_enc = df.copy()

# Encode the `salary` column as an ordinal numeric category
df_enc['salary'] = (
    df_enc['salary'].astype('category')
    .cat.set_categories(['low', 'medium', 'high'])
    .cat.codes
)

# Dummy encode the `department` column
df_enc = pd.get_dummies(df_enc, drop_first=False)

# Display the new dataframe
df_enc.head()

# Create a heatmap to visualize how correlated variables are
plt.figure(figsize=(8, 6))
sns.heatmap(df_enc[['satisfaction_level', 'last_evaluation', 'number_project', 'average_monthly_hours', 'tenure']]
            .corr(), annot=True, cmap="crest")
plt.title('Heatmap of the dataset')
plt.show()

# Create a stacked bart plot to visualize number of employees across department, comparing those who left with those who didn't
pd.crosstab(df['department'], df['left']).plot(kind ='bar',color='mr')
plt.title('Counts of employees who left versus stayed across department')
plt.legend(title='Left', labels=['Stayed', 'Left'])
plt.ylabel('Employee count')
plt.xlabel('Department')
plt.show()

# Since logistic regression is quite sensitive to outliers, it would be a good idea at this stage to remove the outliers in the `tenure` column that were identified earlier.

df_log_reg = df_enc[(df_enc['tenure'] >= lower_limit) & (df_enc['tenure'] <= upper_limit)]

df_log_reg.head()

# Compute the Confusion Matrix
log_cm = confusion_matrix(y_test_log_reg, y_pred)

# Compute Display for conbfusion matrix
log_disp = ConfusionMatrixDisplay(confusion_matrix = log_cm, display_labels = log_clf.classes_)

# Plot the confusion matrix
log_disp.plot(values_format='')

# Display plot
plt.show()

# Compute Classification Report for logistic regression model
target_names = ['Predicted would not leave', 'Predicted would leave']
print(classification_report(y_test_log_reg, y_pred, target_names=target_names))

tree1.fit(X_train_tree1, y_train_tree1)

tree1.best_params_

# Best AUC score
tree1.best_score_

tree1_cv_results = make_results('decision tree cv', tree1, 'roc_auc')
tree1_cv_results

# Check best AUC score on CV
rf1.best_score_

# Check best params
rf1.best_params_

# Get all CV scores
rf1_cv_results = make_results('random forest cv', rf1, 'roc_auc')
print(tree1_cv_results)
print(rf1_cv_results)

# Get predictions on test data
rf1_test_scores = get_scores('random forest1 test', rf1, X_test_tree1, y_test_tree1)
rf1_test_scores

# Generate array of values for confusion matrix
preds_rf1 = rf1.best_estimator_.predict(X_test_tree1)
cm_rf1 = confusion_matrix(y_test_tree1, preds_rf1, labels=rf1.classes_)

# Plot confusion matrix
disp_rf1 = ConfusionMatrixDisplay(confusion_matrix=cm_rf1,
                             display_labels=rf1.classes_)
disp_rf1.plot(values_format='')

# Get feature importances
feat_impt = rf1.best_estimator_.feature_importances_

# Get indices of top 10 features
ind = np.argpartition(rf1.best_estimator_.feature_importances_, -10)[-10:]

# Get column labels of top 10 features 
feat = X.columns[ind]

# Filter `feat_impt` to consist of top 10 feature importances
feat_impt = feat_impt[ind]

y_df1 = pd.DataFrame({"feature":feat,"importance":feat_impt})
y_sort_df1 = y_df1.sort_values("importance")

# Plot the feature importances
fig = plt.figure()
ax1 = fig.add_subplot(111)

y_sort_df1.plot(kind='barh',ax=ax1,x="feature",y="importance")

ax1.set_title("Random Forest: Feature Importances for Employee Leaving", fontsize=12)
ax1.set_ylabel("feature")
ax1.set_xlabel("importance")

plt.show()

# Drop `satisfaction_level` and save resulting dataframe in new variable
df2 = df_enc.drop('satisfaction_level', axis=1)

# Create `overworked` column. For now, it's identical to average monthly hours.
df2['overworked'] = df2['average_monthly_hours']

# Inspect max and min average monthly hours values
print('Max hours:', df2['overworked'].max())
print('Min hours:', df2['overworked'].min())

# Define `overworked` as working > 175 hrs/week
df2['overworked'] = (df2['overworked'] > 175).astype(int)

# Display first few rows of new column
df2['overworked'].head()

%%time
tree2.fit(X_train_tree2, y_train_tree2)

# Check best params
tree2.best_params_

# Check best AUC score on CV
tree2.best_score_

# Get all CV scores
tree2_cv_results = make_results('decision tree2 cv', tree2, 'roc_auc')
print(tree1_cv_results)
print(tree2_cv_results)

# Check best params
rf2.best_params_

# Check best AUC score on CV
rf2.best_score_

# Get all CV scores
rf2_cv_results = make_results('random forest2 cv', rf2, 'roc_auc')
print(tree2_cv_results)
print(rf2_cv_results)

# Get predictions on test data
rf2_test_scores = get_scores('random forest2 test', rf2, X_test_tree2, y_test_tree2)
rf2_test_scores

# Generate array of values for confusion matrix
preds_rf2 = rf2.best_estimator_.predict(X_test_tree2)
cm_rf2 = confusion_matrix(y_test_tree2, preds_rf2, labels=rf2.classes_)

# Plot confusion matrix
disp_rf2 = ConfusionMatrixDisplay(confusion_matrix=cm_rf2,
                             display_labels=rf2.classes_)
disp_rf2.plot(values_format='')

# Plot the tree
plt.figure(figsize=(85,20))
plot_tree(tree2.best_estimator_, max_depth=6, fontsize=14, feature_names=X.columns, 
          class_names={0:'stayed', 1:'left'}, filled=True)
plt.show()

#tree2_importances = pd.DataFrame(tree2.best_estimator_.feature_importances_, columns=X.columns)
tree2_importances = pd.DataFrame(tree2.best_estimator_.feature_importances_, 
                                 columns=['gini_importance'], 
                                 index=X.columns
                                )
tree2_importances = tree2_importances.sort_values(by='gini_importance', ascending=False)

# Only extract the features with importances > 0
tree2_importances = tree2_importances[tree2_importances['gini_importance'] != 0]
tree2_importances

sns.barplot(data=tree2_importances, x="gini_importance", y=tree2_importances.index, orient='h')
plt.title("Decision Tree: Feature Importances for Employee Leaving", fontsize=12)
plt.ylabel("Feature")
plt.xlabel("Importance")
plt.show()

# Get feature importances
feat_impt = rf2.best_estimator_.feature_importances_

# Get indices of top 10 features
ind = np.argpartition(rf2.best_estimator_.feature_importances_, -10)[-10:]

# Get column labels of top 10 features 
feat = X.columns[ind]

# Filter `feat_impt` to consist of top 10 feature importances
feat_impt = feat_impt[ind]

y_df2 = pd.DataFrame({"feature":feat,"importance":feat_impt})
y_sort_df2 = y_df2.sort_values("importance")

# Plot the feature importances
fig = plt.figure()
ax1 = fig.add_subplot(111)

y_sort_df2.plot(kind='barh',ax=ax1,x="feature",y="importance")

ax1.set_title("Random Forest: Feature Importances for Employee Leaving", fontsize=12)
ax1.set_ylabel("feature")
ax1.set_xlabel("importance")

plt.show()

y_sort_df2

# True labels
y_true = [0, 0, 1, 1]

# Predicted probabilities
y_scores = [0.1, 0.4, 0.35, 0.8]

# Compute AUC
auc_score = roc_auc_score(y_true, y_scores)
print(f"AUC Score: {auc_score:.2f}")

# Check best params
rf2.best_params_

# This is only needed if you want to save the model as a JSON file
# Skip this step if you already have the metadata saved

best_score = xgb_model1_fit.best_score_
cv_results = xgb_model1_fit.cv_results_
scorer = xgb_model1_fit.scorer_
refit_time = xgb_model1_fit.refit_time_

metadata = {
    'best_params': xgb_model1_fit.best_params_,
    'best_score': xgb_model1_fit.best_score_,
    'cv_results': xgb_model1_fit.cv_results_,
    'scorer': xgb_model1_fit.scorer_,
    'refit_time': xgb_model1_fit.refit_time_,
    'n_splits': xgb_model1_fit.n_splits_
}

# Save the dictionary using pickle
#with open("xgb_model1_metadata.pkl", "wb") as f:
    #pickle.dump(metadata, f)